{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Demo: Clarify and dense format JSONLines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100   165  100   165    0     0   2323      0 --:--:-- --:--:-- --:--:--  2323\n",
      "100   298    0   298    0     0   2422      0 --:--:-- --:--:-- --:--:--  2422\n",
      "{\"status\":\"error\",\"message\":\"Unauthenticated\",\"desc\":\"You should authenticate (may use mwinit)\",\"step_up_methods\":[{\"cap_name\":\"Midway Authentication\",\"cap_guid\":\"com.amazon.aea.midway.LDAPSuccess\",\"cap_url\":\"https://midway-auth.amazon.com/login\",\"cap_display_string\":\"Please login with Midway.\"}]}"
     ]
    }
   ],
   "source": [
    "!curl -OL https://w.amazon.com/bin/download/Deep-engine/Thundera/Onboarding/MinimumBringYourOwnContainer/WebHome/byoc.tar.gz\n",
    "\n",
    "    \n",
    "!cat byoc.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker import Session\n",
    "session = Session()\n",
    "bucket = session.default_bucket()\n",
    "prefix = 'sagemaker/DEMO-sagemaker-clarify'\n",
    "region = session.boto_region_name\n",
    "# Define IAM role\n",
    "from sagemaker import get_execution_role\n",
    "from sagemaker.s3 import S3Uploader\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import urllib\n",
    "import os\n",
    "\n",
    "role = get_execution_role()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model\n",
    "\n",
    "Please make sure that you have the [simple-star-rating image](https://w.amazon.com/bin/view/Deep-engine/Thundera/Onboarding/MinimumBringYourOwnContainer/#HBYOC) built & pushed to the same region/account that runs this notebook."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train a dummy model,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sagemaker\n",
    "\n",
    "role = sagemaker.get_execution_role()\n",
    "session = sagemaker.Session()\n",
    "\n",
    "account = session.boto_session.client(\"sts\").get_caller_identity()[\"Account\"]\n",
    "region = session.boto_session.region_name\n",
    "image = f\"{account}.dkr.ecr.{region}.amazonaws.com/simple-star-rating:latest\"\n",
    "\n",
    "estimator = sagemaker.estimator.Estimator(\n",
    "    image_uri=image,\n",
    "    role=role,\n",
    "    instance_count=1,\n",
    "    instance_type=\"ml.c4.xlarge\",\n",
    "    sagemaker_session=session,\n",
    ")\n",
    "\n",
    "estimator.fit()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create SageMaker model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = 'simple-start-rating-model'\n",
    "model = estimator.create_model(name=model_name)\n",
    "container_def = model.prepare_container_def()\n",
    "session.create_model(model_name,\n",
    "                     role,\n",
    "                     container_def)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bias analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker import clarify\n",
    "clarify_processor = clarify.SageMakerClarifyProcessor(role=role,\n",
    "                                                      instance_count=1,\n",
    "                                                      instance_type='ml.c4.xlarge',\n",
    "                                                      sagemaker_session=session)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is a workaround. TODO: Remove this cell and next cell when [the JSONLines fix](https://sim.amazon.com/issues/P43801935) is released (tracking SIM: https://sim.amazon.com/issues/P43584479)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# patch image in Thundera devo account. It is the initial launch image plus the JSONLines bug fix\n",
    "clarify_processor.image_uri = \"678264136642.dkr.ecr.us-east-1.amazonaws.com/sagemaker-clarify-processing:1.0_jsonlines_patch\"\n",
    "\n",
    "# new image with the JSONLines bug fix, to be released soon\n",
    "#clarify_processor.image_uri = \"678264136642.dkr.ecr.us-east-1.amazonaws.com/sagemaker-clarify-processing:f988900c\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'s3://sagemaker-us-east-1-617993403527/sagemaker/DEMO-sagemaker-clarify/test_data_for_bias_analysis.jsonl'"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bias_dataset_uri = S3Uploader.upload('test_data_for_bias_analysis.jsonl', 's3://{}/{}'.format(bucket, prefix))\n",
    "bias_dataset_uri"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "NOTE: The order of headers is important\n",
    "* The header of label must be the last one\n",
    "* The order of feature headers should be the same as the order of features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "bias_report_output_path = 's3://{}/{}/clarify-bias'.format(bucket, prefix)\n",
    "bias_data_config = clarify.DataConfig(s3_data_input_path=bias_dataset_uri,\n",
    "                                      s3_output_path=bias_report_output_path,\n",
    "                                      label='star_rating',\n",
    "                                      features='features',\n",
    "                                      headers=['review_body','product_category','star_rating'],\n",
    "                                      dataset_type='application/jsonlines')\n",
    "\n",
    "model_config = clarify.ModelConfig(model_name=model_name,\n",
    "                                   instance_type='ml.c5.xlarge',\n",
    "                                   instance_count=1,\n",
    "                                   accept_type='application/jsonlines',\n",
    "                                   content_type='application/jsonlines',\n",
    "                                   content_template='{\"features\":$features}')\n",
    "\n",
    "predictions_config = clarify.ModelPredictedLabelConfig(\n",
    "    label=\"predicted_label\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "bias_config = clarify.BiasConfig(label_values_or_threshold=[3],\n",
    "                                facet_name='product_category',\n",
    "                                facet_values_or_threshold=['Digital_Software'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Job Name:  Clarify-Bias-2021-01-27-11-32-23-573\n",
      "Inputs:  [{'InputName': 'dataset', 'AppManaged': False, 'S3Input': {'S3Uri': 's3://sagemaker-us-east-1-617993403527/sagemaker/DEMO-sagemaker-clarify/test_data_for_bias_analysis.jsonl', 'LocalPath': '/opt/ml/processing/input/data', 'S3DataType': 'S3Prefix', 'S3InputMode': 'File', 'S3DataDistributionType': 'FullyReplicated', 'S3CompressionType': 'None'}}, {'InputName': 'analysis_config', 'AppManaged': False, 'S3Input': {'S3Uri': 's3://sagemaker-us-east-1-617993403527/Clarify-Bias-2021-01-27-11-32-23-573/input/analysis_config/analysis_config.json', 'LocalPath': '/opt/ml/processing/input/config', 'S3DataType': 'S3Prefix', 'S3InputMode': 'File', 'S3DataDistributionType': 'FullyReplicated', 'S3CompressionType': 'None'}}]\n",
      "Outputs:  [{'OutputName': 'analysis_result', 'AppManaged': False, 'S3Output': {'S3Uri': 's3://sagemaker-us-east-1-617993403527/sagemaker/DEMO-sagemaker-clarify/clarify-bias', 'LocalPath': '/opt/ml/processing/output', 'S3UploadMode': 'EndOfJob'}}]\n",
      "..................................\u001b[34mINFO:sagemaker-clarify-processing:Starting SageMaker Clarify Processing job\u001b[0m\n",
      "\u001b[34mINFO:analyzer.data_loading.data_loader_util:Analysis config path: /opt/ml/processing/input/config\u001b[0m\n",
      "\u001b[34mINFO:analyzer.data_loading.data_loader_util:Analysis result path: /opt/ml/processing/output\u001b[0m\n",
      "\u001b[34mINFO:analyzer.data_loading.data_loader_util:This host is algo-1.\u001b[0m\n",
      "\u001b[34mINFO:analyzer.data_loading.data_loader_util:This host is the leader.\u001b[0m\n",
      "\u001b[34mINFO:analyzer.data_loading.data_loader_util:Number of hosts in the cluster is 1.\u001b[0m\n",
      "\u001b[34mINFO:sagemaker-clarify-processing:Running Python / Pandas based analyzer.\u001b[0m\n",
      "\u001b[34mINFO:analyzer.data_loading.data_loader_util:Dataset uri: /opt/ml/processing/input/data\u001b[0m\n",
      "\u001b[34mINFO:analyzer.data_loading.data_loader_util:Dataset type: application/jsonlines\u001b[0m\n",
      "\u001b[34mINFO:analyzer.util:predictor label configuration will be used as probability configuration\u001b[0m\n",
      "\u001b[34mINFO:analyzer.predictor:Spinning up shadow endpoint\u001b[0m\n",
      "\u001b[34mINFO:sagemaker:Creating endpoint-config with name sagemaker-clarify-endpoint-config-1611747473-b0d1\u001b[0m\n",
      "\u001b[34mINFO:analyzer.predictor:Creating endpoint sagemaker-clarify-endpoint-1611747473-dc40\u001b[0m\n",
      "\u001b[34mINFO:sagemaker-clarify-processing:======================================\u001b[0m\n",
      "\u001b[34mINFO:sagemaker-clarify-processing:Calculating post-training bias metrics\u001b[0m\n",
      "\u001b[34mINFO:sagemaker-clarify-processing:======================================\u001b[0m\n",
      "\u001b[34mINFO:sagemaker-clarify-processing:Getting predictions from the endpoint\u001b[0m\n",
      "\u001b[34mINFO:analyzer.predictor:Checking endpoint status\u001b[0m\n",
      "\u001b[34mINFO:analyzer.predictor:Endpoint is in service after 421 seconds\u001b[0m\n",
      "\u001b[34mINFO:analyzer.predictor:Prediction batch size is initialized with 139810\u001b[0m\n",
      "\u001b[34mINFO:smclarify.bias.metrics.common:data uniqueness fraction: 1.0\u001b[0m\n",
      "\u001b[34mINFO:smclarify.bias.metrics.common:data uniqueness fraction: 1.0\u001b[0m\n",
      "\u001b[34mINFO:smclarify.bias.metrics.common:data uniqueness fraction: 1.0\u001b[0m\n",
      "\u001b[34mINFO:smclarify.bias.metrics.common:data uniqueness fraction: 1.0\u001b[0m\n",
      "\u001b[34mINFO:smclarify.bias.metrics.common:data uniqueness fraction: 1.0\u001b[0m\n",
      "\u001b[34mERROR:smclarify.bias.report:CDDPL metrics failed\u001b[0m\n",
      "\u001b[34mTraceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/report.py\", line 294, in _categorical_metric_call_wrapper\n",
      "    group_variable=group_variable,\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/metrics/__init__.py\", line 26, in call_metric\n",
      "    return metric(**{key: kwargs[key] for key in inspect.signature(metric).parameters.keys()})\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/metrics/posttraining.py\", line 275, in CDDPL\n",
      "    return common.CDD(feature, sensitive_facet_index, positive_predicted_label_index, group_variable)\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/metrics/common.py\", line 80, in CDD\n",
      "    raise ValueError(\"Group variable is empty or not provided\")\u001b[0m\n",
      "\u001b[34mValueError: Group variable is empty or not provided\u001b[0m\n",
      "\u001b[34mERROR:smclarify.bias.report:FT metrics failed\u001b[0m\n",
      "\u001b[34mTraceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/report.py\", line 294, in _categorical_metric_call_wrapper\n",
      "    group_variable=group_variable,\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/metrics/__init__.py\", line 26, in call_metric\n",
      "    return metric(**{key: kwargs[key] for key in inspect.signature(metric).parameters.keys()})\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/metrics/posttraining.py\", line 357, in FT\n",
      "    raise ValueError(\"FlipTest does not support non-numeric columns\")\u001b[0m\n",
      "\u001b[34mValueError: FlipTest does not support non-numeric columns\u001b[0m\n",
      "\u001b[34mINFO:analyzer.predictor:Stop using endpoint: sagemaker-clarify-endpoint-1611747473-dc40\u001b[0m\n",
      "\u001b[34mINFO:sagemaker:Deleting endpoint configuration with name: sagemaker-clarify-endpoint-config-1611747473-b0d1\u001b[0m\n",
      "\u001b[34mINFO:sagemaker:Deleting endpoint with name: sagemaker-clarify-endpoint-1611747473-dc40\u001b[0m\n",
      "\u001b[34mINFO:analyzer.predictor:Model endpoint delivered 4.28179 requests per second and a total of 1 requests over 0 seconds\u001b[0m\n",
      "\u001b[34mINFO:analyzer.predictor:Delivered 1 predict calls with a total of 2 examples.\u001b[0m\n",
      "\u001b[34mINFO:analyzer.predictor:Stop using endpoint: None\u001b[0m\n",
      "\u001b[34mINFO:analyzer.predictor:Delivered 1 predict calls with a total of 2 examples.\u001b[0m\n",
      "\u001b[34mINFO:sagemaker-clarify-processing:=====================================\u001b[0m\n",
      "\u001b[34mINFO:sagemaker-clarify-processing:Calculating pre-training bias metrics\u001b[0m\n",
      "\u001b[34mINFO:sagemaker-clarify-processing:=====================================\u001b[0m\n",
      "\u001b[34mINFO:smclarify.bias.metrics.common:data uniqueness fraction: 1.0\u001b[0m\n",
      "\u001b[34mINFO:smclarify.bias.metrics.common:data uniqueness fraction: 1.0\u001b[0m\n",
      "\u001b[34mINFO:smclarify.bias.metrics.common:data uniqueness fraction: 1.0\u001b[0m\n",
      "\u001b[34mERROR:smclarify.bias.report:CDDL metrics failed\u001b[0m\n",
      "\u001b[34mTraceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/report.py\", line 294, in _categorical_metric_call_wrapper\n",
      "    group_variable=group_variable,\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/metrics/__init__.py\", line 26, in call_metric\n",
      "    return metric(**{key: kwargs[key] for key in inspect.signature(metric).parameters.keys()})\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/metrics/pretraining.py\", line 195, in CDDL\n",
      "    return common.CDD(feature, sensitive_facet_index, positive_label_index, group_variable)\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/metrics/common.py\", line 80, in CDD\n",
      "    raise ValueError(\"Group variable is empty or not provided\")\u001b[0m\n",
      "\u001b[34mValueError: Group variable is empty or not provided\u001b[0m\n",
      "\u001b[34mERROR:smclarify.bias.report:JS metrics failed\u001b[0m\n",
      "\u001b[34mTraceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/report.py\", line 294, in _categorical_metric_call_wrapper\n",
      "    group_variable=group_variable,\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/metrics/__init__.py\", line 26, in call_metric\n",
      "    return metric(**{key: kwargs[key] for key in inspect.signature(metric).parameters.keys()})\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/metrics/pretraining.py\", line 113, in JS\n",
      "    raise ValueError(\"No instance of common facet found, dataset may be too small\")\u001b[0m\n",
      "\u001b[34mValueError: No instance of common facet found, dataset may be too small\u001b[0m\n",
      "\u001b[34mERROR:smclarify.bias.report:KL metrics failed\u001b[0m\n",
      "\u001b[34mTraceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/report.py\", line 294, in _categorical_metric_call_wrapper\n",
      "    group_variable=group_variable,\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/metrics/__init__.py\", line 26, in call_metric\n",
      "    return metric(**{key: kwargs[key] for key in inspect.signature(metric).parameters.keys()})\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/metrics/pretraining.py\", line 91, in KL\n",
      "    raise ValueError(\"No instance of common facet found, dataset may be too small\")\u001b[0m\n",
      "\u001b[34mValueError: No instance of common facet found, dataset may be too small\u001b[0m\n",
      "\u001b[34mERROR:smclarify.bias.report:KS metrics failed\u001b[0m\n",
      "\u001b[34mTraceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/report.py\", line 294, in _categorical_metric_call_wrapper\n",
      "    group_variable=group_variable,\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/metrics/__init__.py\", line 26, in call_metric\n",
      "    return metric(**{key: kwargs[key] for key in inspect.signature(metric).parameters.keys()})\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/metrics/pretraining.py\", line 175, in KS\n",
      "    return LP_norm(label, sensitive_facet_index, np.inf)\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/metrics/pretraining.py\", line 141, in LP_norm\n",
      "    raise ValueError(\"No instance of common facet found, dataset may be too small\")\u001b[0m\n",
      "\u001b[34mValueError: No instance of common facet found, dataset may be too small\u001b[0m\n",
      "\u001b[34mERROR:smclarify.bias.report:LP metrics failed\u001b[0m\n",
      "\u001b[34mTraceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/report.py\", line 294, in _categorical_metric_call_wrapper\n",
      "    group_variable=group_variable,\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/metrics/__init__.py\", line 26, in call_metric\n",
      "    return metric(**{key: kwargs[key] for key in inspect.signature(metric).parameters.keys()})\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/metrics/pretraining.py\", line 132, in LP\n",
      "    return LP_norm(label, sensitive_facet_index, 2)\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/metrics/pretraining.py\", line 141, in LP_norm\n",
      "    raise ValueError(\"No instance of common facet found, dataset may be too small\")\u001b[0m\n",
      "\u001b[34mValueError: No instance of common facet found, dataset may be too small\u001b[0m\n",
      "\u001b[34mERROR:smclarify.bias.report:TVD metrics failed\u001b[0m\n",
      "\u001b[34mTraceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/report.py\", line 294, in _categorical_metric_call_wrapper\n",
      "    group_variable=group_variable,\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/metrics/__init__.py\", line 26, in call_metric\n",
      "    return metric(**{key: kwargs[key] for key in inspect.signature(metric).parameters.keys()})\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/metrics/pretraining.py\", line 158, in TVD\n",
      "    Lp_res = LP_norm(label, sensitive_facet_index, 1)\n",
      "  File \"/usr/local/lib/python3.7/site-packages/smclarify/bias/metrics/pretraining.py\", line 141, in LP_norm\n",
      "    raise ValueError(\"No instance of common facet found, dataset may be too small\")\u001b[0m\n",
      "\u001b[34mValueError: No instance of common facet found, dataset may be too small\u001b[0m\n",
      "\u001b[34m[NbConvertApp] Converting notebook /opt/ml/processing/output/report.ipynb to html\u001b[0m\n",
      "\u001b[34m[NbConvertApp] Writing 277667 bytes to /opt/ml/processing/output/report.html\u001b[0m\n",
      "\u001b[34mINFO:analyzer.report:HTML report '/opt/ml/processing/output/report.html' generated successfully.\u001b[0m\n",
      "\u001b[34mLoading pages (1/6)\u001b[0m\n",
      "\u001b[34m[>                                                           ] 0%#015[======>                                                     ] 10%#015[==========================>                                 ] 44%#015[============================>                               ] 48%#015[======================================================>     ] 90%#015[============================================================] 100%#015Counting pages (2/6)                                               \u001b[0m\n",
      "\u001b[34m[============================================================] Object 1 of 1#015Resolving links (4/6)                                                       \u001b[0m\n",
      "\u001b[34m[============================================================] Object 1 of 1#015Loading headers and footers (5/6)                                           \u001b[0m\n",
      "\u001b[34mPrinting pages (6/6)\u001b[0m\n",
      "\u001b[34m[>                                                           ] Preparing#015[============================================================] Page 1 of 1#015Done                                                                      \u001b[0m\n",
      "\u001b[34mINFO:analyzer.report:PDF report '/opt/ml/processing/output/report.pdf' generated successfully.\u001b[0m\n",
      "\u001b[34mINFO:sagemaker-clarify-processing:Collected analyses: \u001b[0m\n",
      "\u001b[34m{\n",
      "    \"version\": \"1.0\",\n",
      "    \"post_training_bias_metrics\": {\n",
      "        \"label\": \"star_rating\",\n",
      "        \"facets\": {\n",
      "            \"product_category\": [\n",
      "                {\n",
      "                    \"value_or_threshold\": \"Digital_Software\",\n",
      "                    \"metrics\": [\n",
      "                        {\n",
      "                            \"name\": \"AD\",\n",
      "                            \"description\": \"Accuracy Difference (AD)\",\n",
      "                            \"value\": 0.0\n",
      "                        },\n",
      "                        {\n",
      "                            \"name\": \"CDDPL\",\n",
      "                            \"description\": \"Conditional Demographic Disparity in Predicted Labels (CDDPL)\",\n",
      "                            \"value\": null,\n",
      "                            \"error\": \"Group variable is empty or not provided\"\n",
      "                        },\n",
      "                        {\n",
      "                            \"name\": \"DAR\",\n",
      "                            \"description\": \"Difference in Acceptance Rates (DAR)\",\n",
      "                            \"value\": 1.0\n",
      "                        },\n",
      "                        {\n",
      "                            \"name\": \"DCA\",\n",
      "                            \"description\": \"Difference in Conditional Acceptance (DCA)\",\n",
      "                            \"value\": 1.0\n",
      "                        },\n",
      "                        {\n",
      "                            \"name\": \"DCR\",\n",
      "                            \"description\": \"Difference in Conditional Rejection (DCR)\",\n",
      "                            \"value\": 1.0\n",
      "                        },\n",
      "                        {\n",
      "                            \"name\": \"DI\",\n",
      "                            \"description\": \"Disparate Impact (DI)\",\n",
      "                            \"value\": 0.0\n",
      "                        },\n",
      "                        {\n",
      "                            \"name\": \"DPPL\",\n",
      "                            \"description\": \"Difference in Positive Proportions in Predicted Labels (DPPL)\",\n",
      "                            \"value\": 1.0\n",
      "                        },\n",
      "                        {\n",
      "                            \"name\": \"DRR\",\n",
      "                            \"description\": \"Difference in Rejection Rates (DRR)\",\n",
      "                            \"value\": 1.0\n",
      "                        },\n",
      "                        {\n",
      "                            \"name\": \"FT\",\n",
      "                            \"description\": \"Flip Test (FT)\",\n",
      "                            \"value\": null,\n",
      "                            \"error\": \"FlipTest does not support non-numeric columns\"\n",
      "                        },\n",
      "                        {\n",
      "                            \"name\": \"RD\",\n",
      "                            \"description\": \"Recall Difference (RD)\",\n",
      "                            \"value\": 1.0\n",
      "                        },\n",
      "                        {\n",
      "                            \"name\": \"TE\",\n",
      "                            \"description\": \"Treatment Equality (TE)\",\n",
      "                            \"value\": 0.0\n",
      "                        }\n",
      "                    ]\n",
      "                }\n",
      "            ]\n",
      "        },\n",
      "        \"label_value_or_threshold\": \"(3, 5]\"\n",
      "    },\n",
      "    \"pre_training_bias_metrics\": {\n",
      "        \"label\": \"star_rating\",\n",
      "        \"facets\": {\n",
      "            \"product_category\": [\n",
      "                {\n",
      "                    \"value_or_threshold\": \"Digital_Software\",\n",
      "                    \"metrics\": [\n",
      "                        {\n",
      "                            \"name\": \"CDDL\",\n",
      "                            \"description\": \"Conditional Demographic Disparity in Labels (CDDL)\",\n",
      "                            \"value\": null,\n",
      "                            \"error\": \"Group variable is empty or not provided\"\n",
      "                        },\n",
      "                        {\n",
      "                            \"name\": \"CI\",\n",
      "                            \"description\": \"Class Imbalance (CI)\",\n",
      "                            \"value\": 0.0\n",
      "                        },\n",
      "                        {\n",
      "                            \"name\": \"DPL\",\n",
      "                            \"description\": \"Difference in Positive Proportions in Labels (DPL)\",\n",
      "                            \"value\": 1.0\n",
      "                        },\n",
      "                        {\n",
      "                            \"name\": \"JS\",\n",
      "                            \"description\": \"Jensen-Shannon Divergence (JS)\",\n",
      "                            \"value\": null,\n",
      "                            \"error\": \"No instance of common facet found, dataset may be too small\"\n",
      "                        },\n",
      "                        {\n",
      "                            \"name\": \"KL\",\n",
      "                            \"description\": \"Kullback-Liebler Divergence (KL)\",\n",
      "                            \"value\": null,\n",
      "                            \"error\": \"No instance of common facet found, dataset may be too small\"\n",
      "                        },\n",
      "                        {\n",
      "                            \"name\": \"KS\",\n",
      "                            \"description\": \"Kolmogorov-Smirnov Distance (KS)\",\n",
      "                            \"value\": null,\n",
      "                            \"error\": \"No instance of common facet found, dataset may be too small\"\n",
      "                        },\n",
      "                        {\n",
      "                            \"name\": \"LP\",\n",
      "                            \"description\": \"L-p Norm (LP)\",\n",
      "                            \"value\": null,\n",
      "                            \"error\": \"No instance of common facet found, dataset may be too small\"\n",
      "                        },\n",
      "                        {\n",
      "                            \"name\": \"TVD\",\n",
      "                            \"description\": \"Total Variation Distance (TVD)\",\n",
      "                            \"value\": null,\n",
      "                            \"error\": \"No instance of common facet found, dataset may be too small\"\n",
      "                        }\n",
      "                    ]\n",
      "                }\n",
      "            ]\n",
      "        },\n",
      "        \"label_value_or_threshold\": \"(3, 5]\"\n",
      "    }\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34mINFO:analyzer.system_util:exit_message: Completed: SageMaker XAI Analyzer ran successfully\u001b[0m\n",
      "\u001b[34m-------!\u001b[0m\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clarify_processor.run_bias(data_config=bias_data_config,\n",
    "                           bias_config=bias_config,\n",
    "                           model_config=model_config,\n",
    "                           model_predicted_label_config=predictions_config,\n",
    "                           pre_training_methods='all',\n",
    "                           post_training_methods='all')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Explainability analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'s3://sagemaker-us-east-1-617993403527/sagemaker/DEMO-sagemaker-clarify/test_data_for_explainability_analysis.jsonl'"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "explainability_dataset_uri = S3Uploader.upload('test_data_for_explainability_analysis.jsonl', 's3://{}/{}'.format(bucket, prefix))\n",
    "explainability_dataset_uri"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "NOTE: The format of baseline samples should be the same as dataset samples. By definition, baseline should either be a S3 URI to a data file that includes a list of samples, or an in-place list of samples. In this case we chose the latter, and only put a single sample to the list."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "shap_config = clarify.SHAPConfig(baseline=[{\"features\":[\"ok\", \"Digital_Software\"]}],\n",
    "                                 num_samples=10,\n",
    "                                 agg_method='mean_abs')\n",
    "\n",
    "explainability_output_path = 's3://{}/{}/clarify-explainability'.format(bucket, prefix)\n",
    "explainability_data_config = clarify.DataConfig(s3_data_input_path=explainability_dataset_uri,\n",
    "                                s3_output_path=explainability_output_path,\n",
    "                                features='features',\n",
    "                                headers=['review_body','product_category'],\n",
    "                                dataset_type='application/jsonlines')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Job Name:  Clarify-Explainability-2021-01-27-11-45-44-131\n",
      "Inputs:  [{'InputName': 'dataset', 'AppManaged': False, 'S3Input': {'S3Uri': 's3://sagemaker-us-east-1-617993403527/sagemaker/DEMO-sagemaker-clarify/test_data_for_explainability_analysis.jsonl', 'LocalPath': '/opt/ml/processing/input/data', 'S3DataType': 'S3Prefix', 'S3InputMode': 'File', 'S3DataDistributionType': 'FullyReplicated', 'S3CompressionType': 'None'}}, {'InputName': 'analysis_config', 'AppManaged': False, 'S3Input': {'S3Uri': 's3://sagemaker-us-east-1-617993403527/Clarify-Explainability-2021-01-27-11-45-44-131/input/analysis_config/analysis_config.json', 'LocalPath': '/opt/ml/processing/input/config', 'S3DataType': 'S3Prefix', 'S3InputMode': 'File', 'S3DataDistributionType': 'FullyReplicated', 'S3CompressionType': 'None'}}]\n",
      "Outputs:  [{'OutputName': 'analysis_result', 'AppManaged': False, 'S3Output': {'S3Uri': 's3://sagemaker-us-east-1-617993403527/sagemaker/DEMO-sagemaker-clarify/clarify-explainability', 'LocalPath': '/opt/ml/processing/output', 'S3UploadMode': 'EndOfJob'}}]\n",
      "................................\u001b[34mINFO:sagemaker-clarify-processing:Starting SageMaker Clarify Processing job\u001b[0m\n",
      "\u001b[34mINFO:analyzer.data_loading.data_loader_util:Analysis config path: /opt/ml/processing/input/config\u001b[0m\n",
      "\u001b[34mINFO:analyzer.data_loading.data_loader_util:Analysis result path: /opt/ml/processing/output\u001b[0m\n",
      "\u001b[34mINFO:analyzer.data_loading.data_loader_util:This host is algo-1.\u001b[0m\n",
      "\u001b[34mINFO:analyzer.data_loading.data_loader_util:This host is the leader.\u001b[0m\n",
      "\u001b[34mINFO:analyzer.data_loading.data_loader_util:Number of hosts in the cluster is 1.\u001b[0m\n",
      "\u001b[34mINFO:sagemaker-clarify-processing:Running Python / Pandas based analyzer.\u001b[0m\n",
      "\u001b[34mINFO:analyzer.data_loading.data_loader_util:Dataset uri: /opt/ml/processing/input/data\u001b[0m\n",
      "\u001b[34mINFO:analyzer.data_loading.data_loader_util:Dataset type: application/jsonlines\u001b[0m\n",
      "\u001b[34mINFO:analyzer.util:predictor label configuration will be used as probability configuration\u001b[0m\n",
      "\u001b[34mINFO:analyzer.predictor:Spinning up shadow endpoint\u001b[0m\n",
      "\u001b[34mINFO:sagemaker:Creating endpoint-config with name sagemaker-clarify-endpoint-config-1611748251-e903\u001b[0m\n",
      "\u001b[34mINFO:analyzer.predictor:Creating endpoint sagemaker-clarify-endpoint-1611748251-f257\u001b[0m\n",
      "\u001b[34mINFO:explainers.kernel_shap:n_samples 2\u001b[0m\n",
      "\u001b[34mINFO:analyzer.predictor:Checking endpoint status\u001b[0m\n",
      "\u001b[34mINFO:analyzer.predictor:Endpoint is in service after 361 seconds\u001b[0m\n",
      "\u001b[34mINFO:analyzer.shap_analyzer:=====================================================\u001b[0m\n",
      "\u001b[34mINFO:analyzer.shap_analyzer:Shap analyzer: explaining 2 rows, 2 columns...\u001b[0m\n",
      "\u001b[34mINFO:analyzer.shap_analyzer:=====================================================\u001b[0m\n",
      "\u001b[34mINFO:analyzer.predictor:Prediction batch size is initialized with 139810\u001b[0m\n",
      "\u001b[34mN/A% (0 of 2) |                          | Elapsed Time: 0:00:00 ETA:  --:--:--\u001b[0m\n",
      "\u001b[34m100% (2 of 2) |##########################| Elapsed Time: 0:00:00 Time:  0:00:00\u001b[0m\n",
      "\u001b[34mINFO:analyzer.shap_analyzer:getting explanations took 0.34 seconds.\u001b[0m\n",
      "\u001b[34mINFO:analyzer.shap_analyzer:===================================================\u001b[0m\n",
      "\u001b[34mINFO:analyzer.shap_analyzer:converting explanations to tabular took 0.00 seconds.\u001b[0m\n",
      "\u001b[34mINFO:analyzer.shap_analyzer:===================================================\u001b[0m\n",
      "\u001b[34mINFO:analyzer.shap_analyzer:Wrote 2 local explanations to: /opt/ml/processing/output/explanations_shap/out.csv\u001b[0m\n",
      "\u001b[34mINFO:analyzer.shap_analyzer:writing local explanations took 0.00 seconds.\u001b[0m\n",
      "\u001b[34mINFO:analyzer.shap_analyzer:===================================================\u001b[0m\n",
      "\u001b[34mINFO:analyzer.shap_analyzer:aggregating local explanations took 0.00 seconds.\u001b[0m\n",
      "\u001b[34mINFO:analyzer.shap_analyzer:===================================================\u001b[0m\n",
      "\u001b[34mINFO:analyzer.shap_analyzer:Shap analysis finished.\u001b[0m\n",
      "\u001b[34mINFO:analyzer.predictor:Stop using endpoint: sagemaker-clarify-endpoint-1611748251-f257\u001b[0m\n",
      "\u001b[34mINFO:sagemaker:Deleting endpoint configuration with name: sagemaker-clarify-endpoint-config-1611748251-e903\u001b[0m\n",
      "\u001b[34mINFO:sagemaker:Deleting endpoint with name: sagemaker-clarify-endpoint-1611748251-f257\u001b[0m\n",
      "\u001b[34mINFO:analyzer.predictor:Model endpoint delivered 6.58290 requests per second and a total of 3 requests over 0 seconds\u001b[0m\n",
      "\u001b[34mINFO:analyzer.predictor:Delivered 3 predict calls with a total of 7 examples.\u001b[0m\n",
      "\u001b[34m[NbConvertApp] Converting notebook /opt/ml/processing/output/report.ipynb to html\u001b[0m\n",
      "\u001b[34m[NbConvertApp] Writing 295480 bytes to /opt/ml/processing/output/report.html\u001b[0m\n",
      "\u001b[34mINFO:analyzer.report:HTML report '/opt/ml/processing/output/report.html' generated successfully.\u001b[0m\n",
      "\u001b[34mLoading pages (1/6)\u001b[0m\n",
      "\u001b[34m[>                                                           ] 0%#015[======>                                                     ] 10%#015[==========================>                                 ] 44%#015[============================>                               ] 48%#015[=========================================>                  ] 69%#015[======================================================>     ] 90%#015[============================================================] 100%#015Counting pages (2/6)                                               \u001b[0m\n",
      "\u001b[34m[============================================================] Object 1 of 1#015Resolving links (4/6)                                                       \u001b[0m\n",
      "\u001b[34m[============================================================] Object 1 of 1#015Loading headers and footers (5/6)                                           \u001b[0m\n",
      "\u001b[34mPrinting pages (6/6)\u001b[0m\n",
      "\u001b[34m[>                                                           ] Preparing#015[============================================================] Page 1 of 1#015Done                                                                      \u001b[0m\n",
      "\u001b[34mINFO:analyzer.report:PDF report '/opt/ml/processing/output/report.pdf' generated successfully.\u001b[0m\n",
      "\u001b[34mINFO:sagemaker-clarify-processing:Collected analyses: \u001b[0m\n",
      "\u001b[34m{\n",
      "    \"version\": \"1.0\",\n",
      "    \"explanations\": {\n",
      "        \"kernel_shap\": {\n",
      "            \"label0\": {\n",
      "                \"global_shap_values\": {\n",
      "                    \"review_body\": 1.9999999999999996,\n",
      "                    \"product_category\": 4.440892098500626e-16\n",
      "                },\n",
      "                \"expected_value\": 3.0\n",
      "            }\n",
      "        }\n",
      "    }\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34mINFO:analyzer.system_util:exit_message: Completed: SageMaker XAI Analyzer ran successfully\u001b[0m\n",
      "\u001b[34m------!\u001b[0m\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clarify_processor.run_explainability(data_config=explainability_data_config,\n",
    "                                     model_config=model_config,\n",
    "                                     explainability_config=shap_config,\n",
    "                                     model_scores=\"predicted_label\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "instance_type": "ml.t3.medium",
  "kernelspec": {
   "display_name": "Python 3 (Data Science)",
   "language": "python",
   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-east-1:081325390199:image/datascience-1.0"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
